#python3
# -*- coding: utf-8 -*-
"""
Created on 2018/6/29 下午8:04
@author  : XFBY
@Software: PyCharm
"""
# =============================================================================
# 输入关键词（参数：关键字、超时时间、结果页数）-输出搜索标题列表-url列表-正文列表
# =============================================================================

import re
import chardet
import urllib
from lxml import etree
import re
from Get_url import crawler #url
from CxExtractor import Extractor #text
class Get_qa(object):
	def __init__(self):
		pass

	@staticmethod
	def utf8_transfer(strs):
		'''
		utf8编码转换
		'''
		try:
			if isinstance(strs, unicode):
				strs = strs.encode('utf-8')
			elif chardet.detect(strs)['encoding'] == 'GB2312':
				strs = strs.decode("gb2312", 'ignore').encode('utf-8')
			elif chardet.detect(strs)['encoding'] == 'utf-8':
				strs = strs.decode('utf-8', 'ignore').encode('utf-8')
		except Exception, e:
			print 'utf8_transfer error', strs, e
		return strs

	@staticmethod
	def get_title_xpath(Html):
		'''
		用xpath抽取网页Title
		'''
		Html = Get_qa.utf8_transfer(Html)
		Html_encoding = chardet.detect(Html)['encoding']
		page = etree.HTML(Html, parser=etree.HTMLParser(encoding=Html_encoding))
		title = page.xpath('/html/head/title/text()')

		try:
			title = title[0].strip()
		except IndexError:
			print 'Nothing'
		print title


	def _get_title(self,Html):
		'''
		用re抽取网页Title
		'''
		Html = Get_qa.utf8_transfer(Html)
		compile_rule = ur'<title>.*</title>'
		title_list = re.findall(compile_rule, Html)
		if title_list == []:
			title = ''
		else:
			title = title_list[0][7:-8]
		print title

	@staticmethod
	def get_url(keywords, timeout=40, totalpages=1):
		c = crawler(keywords)
		c.set_timeout(timeout)
		c.set_total_pages(totalpages)
		c.run()
		return c.o_urls

	@staticmethod
	def get_title(urls):
		# if not urls[-1] == '/':
		# 	urls = urls+'/'
		try:
			html = urllib.urlopen(urls).read()
			new_html = Get_qa.utf8_transfer(html)
			return Get_qa.get_title_xpath(new_html)
		# get_title(new_html)
		except Exception, e:
			print e
			return 'error_nan'

	@staticmethod
	def get_text(urls):
		ext = Extractor(url=urls, blockSize=5, image=False)
		try:
			s = ext.getContext()
			return s
		except:
			print('extractor text error !')
			return 'error_nan'

	@staticmethod
	def satrt(keywords, timeout=30, totalpages=2):
		titles = [];text=[]
		url_list = Get_qa.get_url(keywords, timeout, totalpages)
		for url in url_list:
			titles.append(Get_qa.get_title(url))
			text.append(Get_qa.get_text(url))
		return titles,url_list,text

#test
if __name__ == '__main__':
	# url = 'https://iask.sina.com.cn/key/573ebac684ae6230f4de50e8.html'
	titles, url_list, text = Get_qa.satrt('冠心病什么症状',totalpages=10)
	print(titles)
	print(12345)
	print(url_list)
	print(23456)
	print(text)
	print("=====================>>/n")
	urls = Get_qa.get_url('冠心病怎么治疗')
	print("=====================>>/n")
	print(urls[0])
	print(Get_qa.get_title(urls[0]))
	print("=====================>>/n")
	print(Get_qa.get_text(urls[0]))